#! /bin/bash
##############################################################
##
## Name		:	muse.funcs
## Author	:	Bradley Atkins
## Description	:	Common functions for muse
## Date		:	20/07/2012
##
##############################################################
##############################################################
##
## Name		:	m_execute
## Author	:	Bradley Atkins
## Description	:	Execute the current script on our cmd line
## Date		:	05/08/2012
## Args		:	Global - M_JOB_FILE (Job control file)
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_execute()
{
	local EXP S=$(hostname) WAIT=N JOB_CNT=0

# TODO change default date format and debug
	
	TS=$(date +"%y/%m/%d-%H:%M:%S")
	#TS=$(date)

	#======================================
	# Get the job details from the control file
	#======================================

	m_read_job_file "${M_JOB_FILE}"

	[[ -n ${M_JOB_ID} ]] || 
		m_fail 1 "Error: ${S} Job ID not retrieved from control file ("${M_JOB_FILE}") (${FUNCNAME})" 
	[[ -n ${M_JOB_NAME} ]] || 
		m_fail 1 "Error: ${S} Job Name not retrieved from control file ("${M_JOB_FILE}") (${FUNCNAME})" 
	[[ -n "${M_JOB_FILE}" ]] || 
		m_fail 1 "Error: ${S} Job file not retrieved from control file ("${M_JOB_FILE}") (${FUNCNAME})" 
	[[ -n "${M_JOB_HOST}" ]] || 
		m_fail 1 "Error: ${S} Run Host not retrieved from control file ("${M_JOB_FILE}") (${FUNCNAME})" 

	#======================================
	# Check if job is part of a group
	#======================================
	if [[ -n ${M_JOB_GRP} ]]
	then
		m_read_grp_file ${M_JOB_GRP} 
		M_GRP_COUNT=$((${M_GRP_COUNT} -1))
		m_write_grp_field ${M_JOB_GRP} ${C_GRP_COUNT} ${M_GRP_COUNT}
	fi

	M_FQ_SCRIPT="$(m_find_file "${M_JOB_NAME}")" || 
		m_fail 1 "Error: m_find_file returned ($?) searching for (${M_JOB_NAME}) (${FUNCNAME})" 
	[[ -f "${M_FQ_SCRIPT}" ]] || m_fail 1 "Error: File not found ("${M_JOB_NAME}") (${FUNCNAME})" 

	## The output file for the script being run
	M_OUT="${M_RESULTS}/${M_JOB_ID}-${M_JOB_NAME}.out"

	#======================================
	# Discover if we are the local or 
	# the remote instance
	#======================================
	if [[ ${M_MUSE_STATUS} == "slave" ]] 
	then
		{
			## Header
			M_JOB_START="$(date)"
			m_print_results_header 

			#======================================
			# Run the job 
			#======================================
			source ${M_FQ_SCRIPT} 
			
			## Trailer
			M_JOB_FINISH="$(date)"
			m_get_duration "${M_JOB_START}" "${M_JOB_FINISH}"
			m_print_results_trailer 

		} 2>&1 | tee "${M_OUT}" ## Results file

		## Tell the master we completed ok
		m_send_msg_to_master "COMPLETED"
	else
		#======================================
		# Listener on Master
		#======================================
		m_create_msg_ctrl_file
		m_create_listener

		#======================================
		# ssh into remote server
		#======================================
		#m_write_job_field ${C_JOB_START} ${TS} "${M_JOB_FILE}"
		m_write_job_field ${C_JOB_START} "$(date)" "${M_JOB_FILE}"
		M_RND=${RANDOM}
		EXP="$(m_set_export_string M_RND M_USR_TAG M_SYS_TAG M_JOB_ID M_MASTER_HOST M_MSG_PIPE)"
		[[ -n "${EXP}" ]] || m_fail 1 "Error: Failure in m_set_export_string (${FUNCNAME})" 

		printf "\t%s\n" "${TS}: - ${M_JOB_NAME}"

		if [[ ${M_JOB_BG} == "Y" ]]
		then
			ssh ${M_USER}@${M_JOB_HOST} "${EXP} muse ${M_JOB_NAME}" > "${M_OUT}" & 
		else
			ssh ${M_USER}@${M_JOB_HOST} "${EXP} muse ${M_JOB_NAME}" > "${M_OUT}" 
		fi

		#======================================
		# If it was a background job record the pid
		# so we can wait for it if needed
		#======================================
		if [[ ${M_JOB_BG} == "Y" ]] 
		then
			m_write_job_field ${C_JOB_PID} $! "${M_JOB_FILE}"

			## Part of a group
			[[ -n ${M_JOB_GRP} ]] && 
				m_write_grp_field ${M_JOB_GRP} ${C_GRP_PID_LIST} "${M_GRP_PID_LIST} $!"
		fi

		#======================================
		# Check for state transitions
		#======================================
		m_test_transitions

		#======================================
		# Loop through any jobs in the pid list
		# that require a wait
		#======================================
		## TODO MOVE THIS INTO MUSE AND TURN INTO A SUDO STATE MACHINE
		for PID in ${M_PID_LIST}
		do
			## TODO CREATE A NON BLOCKING WAIT FUNCTION THAT CAN BE CALLED FROM HERE
			## AND INTEGRATE IT INTO A SUDO STATE MACHINE THAT REACTS TO REMOTE EVENTS.
			wait ${PID}
		done
		M_PID_LIST=
	fi

	return 0
}
##############################################################
##
## Name		:	m_wait
## Author	:	Bradley Atkins
## Description	:	None blocking test for pid. Returns 0
##			if job running. 1 if not.
## Date		:	20/09/2012
## Args		:	1 - PID
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_wait()
{
	[[ $# -eq 1 ]] || m_fail 1 "Error: Usage args (${FUNCNAME})"

	[[ -n $(ps -ef | grep $1 | grep ssh ) ]] && return 0

	return 1
}
##############################################################
##
## Name		:	m_send_msg_to_pipe
## Author	:	Bradley Atkins
## Description	:	Send a msg to a named pipe on the localhost
##			Mainly used to tell the listener to exit
## Date		:	12/09/2012
## Args		:	1 - Status
##			2 - Name of calling function
##			3 - Name of pipe
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_send_msg_to_pipe()
{
	[[ $# -eq 3 ]] || m_fail 1 "Error: Usage (${FUNCNAME})" 

	local MSG="${TS}; $(hostname); Caller=${2}; Status=${1}" PIPE="${3}"

	[[ -p "${PIPE}" ]] || m_fail 1 "Error: Pipe not found (${PIPE}) (${FUNCNAME})" 

	echo "${MSG}" >> "/tmp/${PIPE}" || m_fail 1 "Error: Write to pipe (${PIPE}) failed (${FUNCNAME})" 
}
##############################################################
##
## Name		:	m_send_msg_to_master
## Author	:	Bradley Atkins
## Description	:	Report our status to the master server
## Date		:	12/09/2012
## Args		:	1 - Status
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_send_msg_to_master()
{
	local MSG="${TS}; $(hostname); ${M_MUSE_STATUS}; ${M_FQ_SCRIPT##*/}; Status=${1}"

	echo "${MSG}" | ssh ${M_USER}@${M_MASTER_HOST} "cat >> /tmp/${M_MSG_PIPE}"
}
##############################################################
##
## Name		:	m_create_listener
## Author	:	Bradley Atkins
## Description	:	Instantiate a listener for current job
## Date		:	12/09/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_create_listener()
{
	[[ -n ${C_LISTENER_TIME_OUT} ]] || m_fail 1 "Error: Listener timeout not set (${FUNCNAME})" 

	local LISTENER LPID TO=${C_LISTENER_TIME_OUT}

	## Find the listener
	LISTENER=$(m_find_file u_listener) ||
		m_fail 1 "Error: m_find_file returned ($?) searching for (u_listener) (${FUNCNAME})"
	m_check_file -rx "${LISTENER}" s || m_fail 1 "Error: File validation (${FUNCNAME})" 

	## Launch it
	"${LISTENER}" "${M_MSG_CTL}" ${M_RND} &
	LPID=$!

	## Record its pid and then confirm it's running
	m_write_msg_field ${C_MSG_PID} ${LPID}

	while true
	do
		if [[ $(ls -1 /tmp/"${M_MSG_PIPE}".listening 2>/dev/null) ]]
		then
			rm -f /tmp/"${M_MSG_PIPE}".listening
			break
		fi

		TO=$((TO = ${TO} -1))
		[[ ${TO} -eq 0 ]] && 
			m_fail 1 "Error: Timeout waiting for listener to start (${M_MSG_CTL}) (${FUNCNAME})" 
		
		sleep 0.5
	done
}
##############################################################
##
## Name		:	m_kill_all_listeners
## Author	:	Bradley Atkins
## Description	:	Post run cleanup after a failure
## Date		:	29/09/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_kill_all_listeners()
{
	#TODO CHANGE THIS TO USE THE PIPE TO FORCE AN EXIT - m_send_msg_to_master "COMPLETED"

	local TMPFILE1="$(m_get_tmp_file ${FUNCNAME})" FILE PID PIPE
	[[ -f ${TMPFILE1} ]] || m_fail 1 "Error: Failed to create tmp file (${FUNCNAME})" 

	for FILE in $(ls -1 ${M_CONTROL}/[[:digit:]]*-m_msg_*)
	do
		PID=$(awk '/Pid/{print $2}' FS=":" "${FILE}")
		[[ ${PID} =~ ^[[:digit:]]+$ ]] || continue

		if [[ $(ps -ef | grep u_listener | grep ${PID} | grep -v grep 2>/dev/null) ]]
		then
			kill ${PID}
			echo $PID >> /tmp/pids
		fi
	done
}
##############################################################
##
## Name		:	m_read_msg_ctl_file
## Author	:	Bradley Atkins
## Description	:	Read current message control file as defined
##			by M_MSG_CTL
## Date		:	12/09/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_read_msg_ctl_file()
{
	local TF="$(m_get_tmp_file ${FUNCNAME})"
	m_check_file -fwr "${TF}" s || m_fail 1 "Error: Failed to create tmp file (${FUNCNAME})"

	exec 5<> "${M_MSG_CTL}"
	exec 6<> "${TF}"

	#======================================
	# Read in our control file
	#======================================
	[[ -s "${M_MSG_CTL}" ]] || m_fail 2 "msg control file zero bytes (${M_MSG_CTL})"
	awk '{print $2}' FS=":" <&5 >&6
	exec 6>&-
	exec 5>&-
	exec 6<> "${TF}"
	[[ -s "${TF}" ]] || m_fail 2 "tmp file zero bytes (${TF})"

	{
		read M_MSG_NAME || m_fail 1 "Error: Read 1 (${TF}) (${FUNCNAME})" 	## name of remote job
		read M_MSG_PID || m_fail 1 "Error: Read 2 (${TF}) (${FUNCNAME})" 	## Pid of this listener
		read M_MSG_JOB_CTL_FILE || m_fail 1 "Error: Read 3 (${TF}) (${FUNCNAME})"## Job ctrl file for remote job
		read M_MSG_LOGFILE || m_fail 1 "Error: Read 4 (${TF}) (${FUNCNAME})"     ## Remote host
		read M_MSG_TMPFILE || m_fail 1 "Error: Read 5 (${TF}) (${FUNCNAME})"     ## Remote host
		read M_MSG_HOST || m_fail 1 "Error: Read 6 (${TF}) (${FUNCNAME})"	## Remote host
		read M_MSG_START || m_fail 1 "Error: Read 7 (${TF}) (${FUNCNAME})"	## Start of listen
		read M_MSG_FINISH || m_fail 1 "Error: Read 8 (${TF}) (${FUNCNAME})"	## Finish listen
		read M_MSG_PIPE || m_fail 1 "Error: Read 9 (${TF}) (${FUNCNAME})"	## Name of named pipe
		read M_MSG_STATUS || m_fail 1 "Error: Read 10 (${TF}) (${FUNCNAME})"	## The Remote Job Status
		read M_MSG_CONSTANTS || m_fail 1 "Error: Read 11 (${TF}) (${FUNCNAME})"	## The path to muse.constants
		read M_MSG_WRITE_CNT || m_fail 1 "Error: Read 12 (${TF}) (${FUNCNAME})"	## The number of times this file is written to
		read M_MSG_ACTION || m_fail 1 "Error: Read 13 (${TF}) (${FUNCNAME})"	## The msg Action flag for state machine
	}<&6
	exec 6>&-
}
##############################################################
##
## Name		:	m_write_msg_field
## Author	:	Bradley Atkins
## Description	:	Write to a field in the current msg control file
## Date		:	29/08/2012
## Args		:	1 - Field No
##			2 - Data
##			3 - Optional control file (Default to M_MSG_CTL)
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_write_msg_field()
{
	[[ $# -gt 3 ]] && m_fail 1 "Error: Usage args (${FUNCNAME})"

	local TMPFILE1="$(m_get_tmp_file ${FUNCNAME})" TMP MSG_CTL_FILE="${M_MSG_CTL}"
	[[ $# -eq 3 ]] && MSG_CTL_FILE="${3}"

	m_check_file -fwr "${TMPFILE1}" s || m_fail 1 "Error: File permissions (${TMPFILE1}) (${FUNCNAME})"
	m_check_file -fwr "${MSG_CTL_FILE}" s || m_fail 1 "Error: File permissions (${MSG_CTL_FILE}) (${FUNCNAME})"
	[[ ${1} -gt ${C_MSG_CTRL_CNT} || ${1} -lt 0 ]] && m_fail 1 "Error: Range error (${1}) (${FUNCNAME})" 

	TMP="$(sed -n ''${1}'p' ${MSG_CTL_FILE} | cut -d":" -f1):${2}"
	for s in ${PIPESTATUS[@]}; do [[ $s -eq 0 ]] || m_fail 1 "Error: Pipe failed (${FUNCNAME})" ; done 

	awk -v ln=${1} -v lo="${TMP}" '(NR == ln){print lo;next}{print $0}' "${MSG_CTL_FILE}" > "${TMPFILE1}"

	cp "${TMPFILE1}" "${MSG_CTL_FILE}" || m_fail 1 "Error: Copy failed (${FUNCNAME})" 
}
##############################################################
##
## Name		:	m_create_msg_ctrl_file
## Author	:	Bradley Atkins
## Description	:	Create a msg ctrl file for the listener
##			for the job we are about to kick off. File
##			name is derived from job ctrl file.
## Date		:	12/09/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_create_msg_ctrl_file()
{
	[[ -n ${M_CONTROL} ]] || m_fail 1 "Error: Control files dir not defined (${FUNCNAME})" 
	[[ -n ${M_JOB_FILE} ]] || m_fail 1 "Error: Job file not defined (${FUNCNAME})" 
	[[ -n ${M_CONSTANTS} ]] || m_fail 1 "Error: Constants file not defined (${FUNCNAME})" 
	
	local TMPFILE1="$(m_get_tmp_file u_listener)" 
	m_check_file -fwr "${TMPFILE1}" s || m_fail 1 "Error: Failed to create tmp file (${FUNCNAME})"

	M_MSG_PIPE=$(echo ${M_JOB_FILE##*/} | tr "job" "msg")
	for s in ${PIPESTATUS[@]}; do [[ $s -eq 0 ]] || m_fail 1 "Error: Pipe failed (${FUNCNAME})" ; done 
	
	M_MSG_CTL="${M_CONTROL}/${M_MSG_PIPE}"
	{
		printf "%s\n" "Name:${M_JOB_NAME}"
		printf "%s\n" "Pid:"
		printf "%s\n" "JobFile:${M_JOB_FILE}"
		printf "%s\n" "LogFile:${M_RUN_LOG}"
		printf "%s\n" "TmpFile:${TMPFILE1}"
		printf "%s\n" "JobHost:${M_JOB_HOST}"
		printf "%s\n" "Start:${TS}"
		printf "%s\n" "Finish:"
		printf "%s\n" "Pipe:${M_MSG_PIPE}"
		printf "%s\n" "Status:Pending"
		printf "%s\n" "Constants:${M_CONSTANTS}"
		printf "%s\n" "Writes:1"
		printf "%s\n" "Action:Skip"
	} > "${M_MSG_CTL}" || m_fail 1 "Error: Write failed (${FUNCNAME})" 
}
##############################################################
##
## Name		:	m_test_transitions
## Author	:	Bradley Atkins
## Description	:	Check if we are making a transition from
##			from a single job or group of jobs and if
##			so then check for wait required.
## Date		:	06/09/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_test_transitions()
{
	m_read_job_file "${M_JOB_FILE}"
	if [[ -z ${M_JOB_GRP} ]]
	then
		[[ ${M_JOB_WAIT} == Y ]] && M_PID_LIST=${M_JOB_PID}
		return
	fi

	m_read_grp_file ${M_JOB_GRP} 
	if [[ -n ${M_JOB_GRP} && ${M_GRP_COUNT} -eq 0 ]]
	then
		[[ ${M_GRP_WAIT} == Y ]] && M_PID_LIST=${M_GRP_PID_LIST}
		return
	fi
}
##############################################################
##
## Name		:	m_read_grp_file
## Author	:	Bradley Atkins
## Description	:	Read the fields in a grp file
## Date		:	29/08/2012
## Args		:	INDX of grp file
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_read_grp_file()
{
	[[ $# -eq 1 ]] || m_fail 1 "Error: Usage args (${FUNCNAME})"

	local FQFN TMPFILE="$(m_get_tmp_file ${FUNCNAME})"
	m_check_file -fwr "${TMPFILE}" s || m_fail 1 "Error: Failed to create tmp file (${FUNCNAME})"

	FQFN=$(ls "${M_CONTROL}/${1}-${C_GRP_PREFIX}"*) ||
		m_fail 1 "Error: Failed to find group file (${1}) (${FUNCNAME})" 

	m_check_file -rs "${FQFN}" s || 
		m_fail 1 "Error: File validation (${FNAME}) (${FUNCNAME})" 

	[[ $(wc -l "${FQFN}" | awk '{print $1}') -eq ${C_GRP_CTRL_FCNT} ]] || 
		m_fail 1 "Error: Field Count (${FQFN}) (${FUNCNAME})" 


	awk '{print $2}' FS=":" "${FQFN}" > "${TMPFILE}" 
	{
		read M_GRP_ID || m_fail 1 "Error: Read failed 1 (${FUNCNAME})" 
		read M_GRP_WAIT || m_fail 1 "Error: Read failed 2 (${FUNCNAME})"
		read M_GRP_COUNT || m_fail 1 "Error: Read failed 3 (${FUNCNAME})"
		read M_GRP_PID_LIST || m_fail 1 "Error: Read failed 4 (${FUNCNAME})" 
	} < "${TMPFILE}" 
}
##############################################################
##
## Name		:	m_write_grp_field
## Author	:	Bradley Atkins
## Description	:	Write to a field in the current grp control file
## Date		:	29/08/2012
## Args		:	1 - Job ID, (File index No)
##			2 - Field No
##			3 - Data
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_write_grp_field()
{
	[[ $# -eq 3 ]] || m_fail 1 "Error: Usage args (${FUNCNAME})"

	local TMPFILE1="$(m_get_tmp_file ${FUNCNAME})" TMP FQFN
	m_check_file -fwr "${TMPFILE1}" s || m_fail 1 "Error: Failed to create tmp file (${FUNCNAME})"

	FQFN=$(ls "${M_CONTROL}/${1}-${C_GRP_PREFIX}"*) ||
		m_fail 1 "Error: Failed to find group file (${1}) (${FUNCNAME})" 
	
	[[ ${2} -gt ${C_GRP_CTRL_FCNT} || ${2} -lt 0 ]] && m_fail 1 "Error: Range error (${FUNCNAME})" 

	TMP="$(sed -n ''${2}'p' ${FQFN} | cut -d":" -f1):${3}"
	for s in ${PIPESTATUS[@]}; do [[ $s -eq 0 ]] || m_fail 1 "Error: Pipe failed (${FUNCNAME})" ; done 

	awk -v ln=${2} -v lo="${TMP}" '(NR == ln){print lo;next}{print $0}' "${FQFN}" > "${TMPFILE1}"

	cp "${TMPFILE1}" "${FQFN}"
}
##############################################################
##
## Name		:	m_write_job_field
## Author	:	Bradley Atkins
## Description	:	Write to a field in the current job control file
## Date		:	29/08/2012
## Args		:	1 - Field No
##			2 - Data
##			3 - Job control file
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_write_job_field()
{
	[[ $# -eq 3 ]] || m_fail 1 "Error: Usage args (${FUNCNAME})"

	local TMPFILE1="$(m_get_tmp_file ${FUNCNAME})" TMP JOB_FILE="${3}"
	m_check_file -fwr "${TMPFILE1}" s || m_fail 1 "Error: Failed to create tmp file (${FUNCNAME})"
	m_check_file -fwr "${JOB_FILE}" s || m_fail 1 "Error: Job ctl file validation (${FUNCNAME})"

	[[ (${1} -gt ${C_JOB_CTRL_FCNT}) || (${1} -lt 0) ]] && m_fail 1 "Error: Range error (${FUNCNAME})" 

	TMP="$(sed -n ''${1}'p' ${JOB_FILE} | cut -d":" -f1):${2}"
	for s in ${PIPESTATUS[@]}; do [[ $s -eq 0 ]] || m_fail 1 "Error: Pipe failed (${FUNCNAME})" ; done 

	awk -v ln=${1} -v lo="${TMP}" '(NR == ln){print lo;next}{print $0}' "${JOB_FILE}" > "${TMPFILE1}"

	cp "${TMPFILE1}" "${JOB_FILE}" || m_fail 1 "Error: Copy failed (${FUNCNAME})" 
}
##############################################################
##
## Name		:	m_read_job_file
## Author	:	Bradley Atkins
## Description	:	Read the fields in a job file
## Date		:	29/08/2012
## Args		:	FQFN of job file
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_read_job_file()
{
	local FILE=${1} TF="$(m_get_tmp_file ${FUNCNAME})"
	m_check_file -fwr "${TF}" s || m_fail 1 "Error: Failed to create tmp file (${FUNCNAME})"

	[[ $# -eq 1 ]] || m_fail 1 "Error: Usage args (${FUNCNAME})"
	m_check_file -rs "${FILE}" s || 
		m_fail 1 "Error: File validation (${FILE}) (${FUNCNAME})" 

	[[ $(wc -l "${FILE}" | awk '{print $1}') -eq ${C_JOB_CTRL_FCNT} ]] || 
		m_fail 1 "Error: Field Count (${FILE}) (${FUNCNAME})" 

	#======================================
	# Open and close our files
	#======================================
	exec 5<> "${FILE}"
	exec 6<> "${TF}"
	awk '{print $2}' FS=":" <&5 >&6
	exec 6>&-
	exec 5>&-
	[[ -s "${TF}" ]] || m_fail 2 "tmp file zero bytes (${TF})"
	
	#======================================
	# Read the file
	#======================================
	exec 6<> "${TF}"
	{
		set -a
		read M_JOB_ID || m_fail 1 "Error: Read failed 1 (${FUNCNAME})" 
		read M_JOB_NAME || m_fail 1 "Error: Read failed 2 (${FUNCNAME})"
		read M_JOB_PATH || m_fail 1 "Error: Read failed 3 (${FUNCNAME})"
		read M_JOB_HOST || m_fail 1 "Error: Read failed 4 (${FUNCNAME})"
		read M_JOB_GRP || m_fail 1 "Error: Read failed 5 (${FUNCNAME})"
		read M_JOB_PID || m_fail 1 "Error: Read failed 6 (${FUNCNAME})"
		read M_JOB_START || m_fail 1 "Error: Read failed 7 (${FUNCNAME})"
		read M_JOB_FINISH || m_fail 1 "Error: Read failed 8 (${FUNCNAME})"
		read M_JOB_WAIT || m_fail 1 "Error: Read failed 9 (${FUNCNAME})"
		read M_JOB_BG || m_fail 1 "Error: Read failed 10 (${FUNCNAME})"
		set +a
	}<&6

	exec 6>&-
}
##############################################################
##
## Name		:	m_set_export_string
## Author	:	Bradley Atkins
## Description	:	Create the export string for the ssh to the 
##			target server.
## Date		:	05/08/2012
## Args		:	List of env var names (string types)
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_set_export_string()
{
	[[ $# -ne 0 ]] || return 1

	local STR 

	for VAR in $@
	do
		NAME=${VAR}
		eval VAL=\$${VAR}
		[[ -n ${VAL} ]] || return 2

		NAME="$(echo ${NAME} | sed 's/^M_/P_/g')"

		STR="${STR} ${NAME}=\"${VAL}\""
	done

	echo "${STR}"
}
##############################################################
##
## Name		:	m_parse_muse_script
## Author	:	Bradley Atkins
## Description	:	Find the script we are about to run
##			and verify etc
## Date		:	04/08/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_parse_muse_script()
{
	# TODO CHANGE THE NAME OF THIS FUNCTION IF IT ONLY RETRIEVES THE JOB HOST
	[[ $# -eq 1 ]] || m_fail 1 "Error: Usage args (${FUNCNAME})"

	local FQFN

	#======================================
	# We have been passed an unqualified
	# script name so we need to find it
	#======================================
	FQFN="$(m_find_file "${1}")" ||
		m_fail 1 "Error: m_find_file returned ($?) searching for (${1}) (${FUNCNAME})"
	m_check_file -rx "${FQFN}" s || m_fail 1 "Error: file not readable or executable (${FUNCNAME})"

	## Where are we running? 
	M_JOB_HOST=$(m_get_run_host "${FQFN}")

	[[ -n "${M_JOB_HOST}" ]] || m_fail 1 "Error: Failed to get job host (${FUNCNAME})" 
}
##############################################################
##
## Name		:	m_get_grep_pattern
## Author	:	Bradley Atkins
## Description	:	Retrieve the grep pattern, if set, from 
##			a script header. Assumes readable file
## Date		:	11/09/2012
## Args		:	FQName of script
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_get_grep_pattern()
{
	## Usage
	[[ $# -eq 1 ]] || return 1

	## Parse the M_JOB_HOST from the header
	local TMPSTR1=$(sed -n '/M_GREP_PATTERN/p' "${1}") TMPSTR2 PATTERN

	## Incorrect header format
	[[ -n ${TMPSTR1} ]] || return 2

	## Cut out the pattern
	TMPSTR2=$(echo ${TMPSTR1} | cut -d" " -f3-)
	for s in ${PIPESTATUS[@]}; do [[ $s -eq 0 ]] || return 3 ; done 

	## If name not set just default to constant
	if [[ -z "${TMPSTR2}" ]] 
	then
		PATTERN="${C_GREP_PATTERN}"
	else
		PATTERN="${TMPSTR2}"
	fi

	## Is this a valid pattern?
	TMP=$(egrep -i "${PATTERN}" "${1}") || return 4

	echo "${TMPSTR2}"
}
##############################################################
##
## Name		:	m_get_run_host
## Author	:	Bradley Atkins
## Description	:	Retrieve the run server, if set, from 
##			a script header. Assumes readable file
## Date		:	04/08/2012
## Args		:	FQName of script
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_get_run_host()
{
	[[ $# -eq 1 ]] || return 1

	## Parse the M_JOB_HOST from the header
	local TMPSTR1=$(sed -n '/M_JOB_HOST/p' "${1}") TMPSTR2

	## Malformed header
	[[ -n ${TMPSTR1} ]] || return 2

	## Cut out the server name
	TMPSTR2=$(echo ${TMPSTR1} | cut -d" " -f3)

	## If name not set just default to local host
	if [[ -z "${TMPSTR2}" ]] 
	then
		echo $(hostname)
		return
	fi

	## Got a server name. Is it valid?
	TMPSTR1=$(m_get_host_data "${TMPSTR2}" SNAME) || return 3

	## Unknown job host
	[[ -z "${TMPSTR1}" ]] && return 4

	echo "${TMPSTR1}"
}
##############################################################
##
## Name		:	m_get_host_data
## Author	:	Bradley Atkins
## Description	:	Retrieve server data from m_hosts for
##			curent cluster.
## Date		:	04/08/2012
## Args		:	1 - Server name
##			2 - Data Type <CLUSTER FQNAME SNAME IP OS>
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_get_host_data()
{
	[[ $# -eq 2 ]] || m_fail 1 "Error: Usage args (${FUNCNAME})"

	eval local FI=\$H_${2}

	local FILE1="$(m_get_tmp_file ${FUNCNAME})" DT=
	m_check_file -fwr "${FILE1}" s || m_fail 1 "Error: Failed to create tmp file (${FUNCNAME})"

	#======================================
	# We parse m_hosts to identiy the server
	# Field order is -
	# CLUSTER FQNAME SNAME IP OS
	#======================================

	## Confirm hosts is available and readable 
	## then strip out comments etc
	m_check_file -rfs "${M_HOSTS}" s || 
		m_fail 1 "Error: (${M_HOSTS}) not a readable flat file (${FUNCNAME})"

	m_clean_file "${M_HOSTS}" "${FILE1}"
	
	m_check_file -rfs "${FILE1}" s || 
		m_fail 1 "Error: File validation ("${FILE1}") (${FUNCNAME})"

	## Passing a cluster name instead of a server would break us
	## As this is a coding error then die
	[[ -z $(awk -v s="${1}" -v ci=${H_CLUSTER} ' ($ci == s){print "fail"} ' "${FILE1}") ]] || 
		m_fail 1 "Error: Cluster name passed as server name (${FUNCNAME})"

	## Get the data field from the file
	DT=$(awk -v c="${M_CLUSTER}" -v ci=${H_CLUSTER} -v fi=${FI} -v s="${1}" ' ( ($ci == c) && ($0 ~ s) ){
		print $fi
	} ' "${FILE1}")

	[[ -z "${DT}" ]] && return 1

	echo "${DT}" 
}
##############################################################
##
## Name		:	m_init
## Author	:	Bradley Atkins
## Description	:	Set up our runtime env
## Date		:	22/07/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_init()
{
	[[ $# -eq 0 ]] || m_fail 1 "Error: Usage args (${FUNCNAME})"

	## Check we have set our exported variables so we can create our run file
	[[ -n ${M_SYS_TAG} ]] || m_fail 1 "Error: Sys tag not set (${FUNCNAME})"
	[[ -n ${M_USR_TAG} ]] || m_fail 1 "Error: Usr tag not set (${FUNCNAME})"
	[[ -n ${M_RUN_ROOT} ]] || m_fail 1 "Error: Run root not set (${FUNCNAME})"

	#======================================
	# Set our group id so that all run time
	# files and dirs have the correct group
	#======================================
	mkdir -p "${M_RUN_ROOT}"
	[[ -d "${M_RUN_ROOT}" ]] || 
		m_fail 1 "Error: Failed to create run time root dir (${FUNCNAME}"

	chmod g+s "${M_RUN_ROOT}" || m_fail 1 "Error: Failed chmod of ("${M_RUN_ROOT}") (${FUNCNAME})" 
	chown :${M_GROUP} "${M_RUN_ROOT}" || m_fail 1 "Error: Failed chown of ("${M_RUN_ROOT}") (${FUNCNAME})" 
	
	# TODO DECIDE IF WE NEED THIS IN A REAL QA CLUSTER. NEED TO FINALISE SECURING MUSE
	#sudo chmod g+s "${M_RUN_ROOT}" || m_fail 1 "Error: Failed chmod of ("${M_RUN_ROOT}") (${FUNCNAME})" 
	#sudo chown :${M_GROUP} "${M_RUN_ROOT}" || m_fail 1 "Error: Failed chown of ("${M_RUN_ROOT}") (${FUNCNAME})" 

	#======================================
	# The Persistent System Directories
	# may not exist (first run)
	#======================================
	## For the tag and job control files
	mkdir -p "${M_RUN_ROOT}/m_system"
	[[ -d "${M_RUN_ROOT}/m_system" ]] || 
		m_fail 1 "Error: Failed to create run time system dir (${FUNCNAME}"

	#======================================
	# Create our tagged runtime directories
	#======================================
	mkdir -p "${M_NEWEXP}"
	[[ -d "${M_NEWEXP}" ]] || m_fail 1 "Error: Failed to create run time exp dir (${FUNCNAME}"

	mkdir -p "${M_TMP}"
	[[ -d "${M_TMP}" ]] || m_fail 1 "Error: Failed to create run time tmp dir (${FUNCNAME}"

	mkdir -p "${M_CONTROL}"
	[[ -d "${M_CONTROL}" ]] || m_fail 1 "Error: Failed to create run time control dir (${FUNCNAME}"

	mkdir -p "${M_RESULTS}"
	[[ -d "${M_RESULTS}" ]] || m_fail 1 "Error: Failed to create run time results dir (${FUNCNAME}"

	#======================================
	# Development directory
	#======================================
	mkdir -p "${M_DEV_ROOT}"
	[[ -d "${M_DEV_ROOT}" ]] || 
		m_fail 1 "Error: Failed to create development directory ("${M_DEV_ROOT}") (${FUNCNAME})" 

	#======================================
	# Clear our log and run data
	#======================================
	:>"${M_RUN_LOG}"
	:>"${M_JOB_CTRL}"

	#======================================
	# Initialise the msg queue
	#======================================
	M_MSG_QUEUE="$(m_get_tmp_file msg_queue)"
	m_check_file -fwr "${M_MSG_QUEUE}" s || m_fail 1 "Error: Failed to create msg file (${FUNCNAME})"
}
##############################################################
##
## Name		:	m_get_usr_tag
## Author	:	Bradley Atkins
## Description	:	Get the user tag if set and confirm a 
##			viable directory name.
## Date		:	19/08/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_get_usr_tag()
{
	#======================================
	# If the file is empty or unreadable
	# then no tag set so return
	#======================================

	M_USR_TAG=default

	m_check_file -sr "${M_USR_TAG_FILE}" s || return

	[[ $(wc -l "${M_USR_TAG_FILE}" | cut -d" " -f1) -eq 1 ]] || 
		m_fail 1 "Error: Multi Line tag set (${FUNCNAME})"

	#======================================
	# It exists so we need to confirm it 
	# contains a valid unix directory name
	#======================================
	local TMP=$(cat "${M_USR_TAG_FILE}")
	[[ -n "${TMP}" ]] || return 1

	mkdir "/tmp/${TMP}" 2>/dev/null || m_fail 1 "Error: Invalid tag name set (${TMP}) (${FUNCNAME})" 
	[[ -d "/tmp/${TMP}" ]] || m_fail 1 "Error: failed to catch mkdir failure (${FUNCNAME})" 
	rm -rf "/tmp/${TMP}" || m_fail 1 "Error: Failed to delete tmp tag directory (${FUNCNAME})" 

	#======================================
	# Must be valid if we got this far so
	# set the usr tag
	#======================================
	M_USR_TAG="${TMP}"
}
##############################################################
##
## Name		:	m_validate_environment_variables
## Author	:	Bradley Atkins
## Description	:	Confirm if the vars passed are set
## Date		:	22/07/2012
## Args		:	1 - Set or Unset n or z
##			2 - List of variable names
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##
##############################################################
m_validate_environment_variables()
{
	[[ $# -eq 0 ]] && return 1
	[[ ${1} != "z" && ${1} != "n" ]] && return 2

	local EV RESULT=pass TEST=${1}

	shift

	for EV in $@
	do
		eval [[ -${TEST}  '"${'${EV}'}"' ]] || RESULT=fail
		if [[ ${RESULT} == fail ]]
		then
			echo ${EV}
			return 3
		fi
	done

	return 0
}
##############################################################
##
## Name		:	m_post_run_tasks
## Author	:	Bradley Atkins
## Description	:	Try to perform a tidy up prior to finish
##			of run
## Date		:	09/09/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_post_run_tasks()
{
	#======================================
	# NOTE - No m_fail in here. inifinite recursion
	#======================================

	## No summaries on server end of ssh connection
	[[ ${M_MUSE_STATUS} == "slave" ]] && return 1

	local RESFILE RESLIST OUTFILE TMP GREP_MSGS="${C_GREP_MSGS}" FQFN

	RESLIST=$(echo $(ls -1 ${M_RESULTS}/[[:digit:]]-*.out 2>/dev/null))
	[[ -n ${RESLIST} ]] || return 2 ## No results to parse

	local TMPFILE1="$(m_get_tmp_file ${FUNCNAME})"
	m_check_file -fwr "${TMPFILE1}" s || return 3

	m_log_msg "Executing final wait before post run tasks"
	wait

	local EVDC="${M_RESULTS}/summary.evidence" MSGS="${M_RESULTS}/summary.msgs"
	for RESFILE in ${RESLIST}
	do
		#======================================
		# Parse the results files for evidence 
		# markers, warnings and errors
		#======================================

		TMP=$(basename ${RESFILE})

		if [[ -n $(egrep "${C_GREP_EVIDENCE}" "${RESFILE}" 2>/dev/null) ]]
		then
			{
				printf "%s\n" "${C_BREAK}"
				printf "%s\n" " Summary of marked evidence in - ${TMP}"
				printf "%s\n" "${C_BREAK}"
				awk -v p="$C_EVIDENCE" -v op=0 '($0==p){op=!op;print p;next} op{print}' ${RESFILE}
				printf "%s\n" "${C_BREAK}"
				printf "\n\n"
			} >> "${EVDC}"
		fi

		## Get the grep pattern for this job
		FQFN="$(m_find_file "$(x=$(basename ${RESFILE});y=${x%%.*};echo ${y#*-})")" 
		m_check_file -rf "${FQFN}" s || m_fail 1 "Error:Failed to find file (${1}) (${FUNCNAME})"

		GREP_PATTERN=$(m_get_grep_pattern "${FQFN}")

		## TODO ADD EGREP OPTION AS EXTRA ARG IN HEADER AND SWITCH TO TURN THIS FEATURE OFF
		## grep for msgs of interest
		eval egrep -i ${GREP_PATTERN} "${RESFILE}" > "${TMPFILE1}" 
		if [[ $? -eq 0 ]]
		then
			{
				printf "%s\n" "${C_BREAK}"
				printf "%s\n" " Summary of msgs in - ${TMP}"
				printf "%s\n\n" "${C_BREAK}"
				cat "${TMPFILE1}"
				printf "\n%s\n" "${C_BREAK}"
				printf "\n\n"
			} >> "${MSGS}"
		fi
	done

}
##############################################################
##
## Name		:	m_fail
## Author	:	Bradley Atkins
## Description	:	Handle a runtime failure. Either -
##			Log error and exit
##			Log error and continue
## Date		:	22/07/2012
## Args		:	1 - <exit|fatal exit> 
##			2 -<Message string>
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##
##############################################################
m_fail()
{
	local FAIL_ACTION=${1:-1} M_LAST_FAIL_MSG=${2:-"Failed without message"} MSG

	MSG="${M_LAST_FAIL_MSG}"
	[[ -n ${GE} ]] && MSG="${M_LAST_FAIL_MSG} (GE = ${GE})" ## GE Global error string for extra debug

	## Log it in the run log
	m_log_msg "${M_LAST_FAIL_MSG}" 

	## Echo it out so we see it in the results files
	if [[ ${M_MUSE_STATUS} == "slave" ]]
	then
		printf "%s\n" "${C_ERROR}"
		echo "${M_LAST_FAIL_MSG}"
		printf "%s\n" "${C_ERROR}"
	fi

	STATE="FATAL"
	[[ ${FAIL_ACTION} -eq 1 ]] && STATE="FAILED"
	
	printf "%s\n" "${C_BREAK}"
	printf "%s\n" " Finish:	$(date)"
	printf "\t\t%s\n" " EARLY EXIT DUE TO FAILURE"
	printf "%s\n" "${C_BREAK}"

	if [[ ${M_MUSE_STATUS} == "master" ]] 
	then
		m_post_run_tasks
	else
		m_send_msg_to_master "${STATE}"
	fi

	exit
}
##############################################################
##
## Name		:	m_log_msg
## Author	:	Bradley Atkins
## Description	:	Write a log message
## Date		:	04/08/2012
## Args		:	1 - msg
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_log_msg()
{
	echo "$(date): ${M_MUSE_STATUS} - $(hostname) - ${1}" >> "${M_RUN_LOG}"
}
##############################################################
##
## Name		:	m_prompt_yesno
## Author	:	Bradley Atkins
## Description	:	Prompt the user for a yes or no answer
## Date		:	27/08/2012
## Args		:	1 - Question
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_prompt_yesno()
{
	[[ $# -eq 1 ]] || m_fail 1 "Error: Usage args (${FUNCNAME})"

	printf "\n%s\n" "${1}"
	while true
	do
		read -p "Do you want to continue (Y/N)?" ANSWER
		case ${ANSWER} in
			Y|y)
				return 0;;
			N|n) 
				return 1;;
			*) echo "Please answer y or n.";;
		esac
	done
}
##############################################################
##
## Name		:	m_validate_cmd_line
## Author	:	Bradley Atkins
## Description	:	Validate the command line submitted by the user
## Date		:	21/07/2012
## Args		:	Valid args are -
##			Valid options in arg1
##			Any number of scripts begining m_
##			Any number of suites begining ms_
##			Suites may contain a sub set of valid options
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##
##############################################################
m_validate_cmd_line()
{
	[[ $# -eq 0 ]] && m_fail 1 "Error: Usage args (${FUNCNAME})"
	
	local ARG FILE

	#======================================
	# First deal with any options
	#======================================
	if [[ $1 =~ ^-|^-- ]] 
	then
		local RES=pass

		case ${1} in
			-a|--archive)echo "Option not coded yet";exit 1;;
			-c|--clear) ## Delete all results if no tag set or just the results under tag name passed
				m_remove_run_files
				M_ACTION=NORUN
				if [[ $# -gt 1 ]] 	## Clear results under a tag name
				then
					shift
					m_check_file -d "${M_RUN}/${1}" s
					if [[ $? -eq 0 ]]
					then
						m_prompt_yesno "Delete all results under tag (${1})?"
						if [[ $? -eq 0 ]] 
						then
							rm -rf "${M_RUN}/${1}/" || 
								m_fail 1 "Error: Delete tagged failed (${FUNCNAME})" 
						fi
					else
						printf "\n%s\n\n" "No tagged results found for (${1})"
					fi
				else 			## Clear all results
					m_prompt_yesno "Delete all results?"
					if [[ $? -eq 0 ]]
					then
						if [[ -d "${M_RUN}" ]] 
						then
							rm -rf "${M_RUN}/" 2>/dev/null
						else
							printf "\n%s\n\n" "No results directories found for deletion"
						fi
					fi
				fi	
				return 0
				;;
			-d|--delete) ## Delete the current tag
				m_remove_run_files
				:>"${M_USR_TAG_FILE}"
				M_ACTION=NORUN
				return 0
				;;
			-e|--explain)
				shift
				if [[ $# -gt 0 ]]
				then
					m_process_cmd_list $@
					cat "${M_JOB_CTRL}"
				else
					usage
				fi
				m_remove_run_files
				M_ACTION=NORUN
				return 0
				;;
			-f|--find) ## Find a set of results or all results
				shift
				m_remove_run_files
				if [[ -n ${1} ]] ## User wants a tagged set
				then
					m_check_file -d "${M_RUN}/${1}" s
					if [[ $? -eq 0 ]]
					then
						ls -lR "${M_RUN}/${1}/"*/m_results
					else
						printf "\n%s\n\n" "No tagged results found for (${1})"
					fi
				else ## No tag passed so show all results
					ls -lR "${M_RUN}"/*/*/m_results
				fi
				M_ACTION=NORUN
				return 0
				;;
			-fc|--findc) ## Find a set of control files
				shift
				m_remove_run_files
				if [[ -n ${1} ]] ## User wants a tagged set
				then
					m_check_file -d "${M_RUN}/${1}" s
					if [[ $? -eq 0 ]]
					then
						ls -lR "${M_RUN}/${1}/"*/m_control
					else
						printf "\n%s\n\n" "No tagged files found for (${1})"
					fi
				else ## No tag passed so show all results
					ls -lR "${M_RUN}"/*/*/m_control
				fi
				M_ACTION=NORUN
				return 0
				;;
			-ft|--findt) ## Find a set of tmp files
				shift
				m_remove_run_files
				if [[ -n ${1} ]] ## User wants a tagged set
				then
					m_check_file -d "${M_RUN}/${1}" s
					if [[ $? -eq 0 ]]
					then
						ls -lR "${M_RUN}/${1}/"*/m_tmp
					else
						printf "\n%s\n\n" "No tagged files found for (${1})"
					fi
				else ## No tag passed so show all results
					ls -lR "${M_RUN}"/*/*/m_tmp
				fi
				M_ACTION=NORUN
				return 0
				;;
			-h|--help)
				usage
				exit 0;;
			-l|--list)
				shift
				if [[ $# -ne 0 ]]
				then
					m_process_cmd_list $@
					awk '($1 ~ "ID"){printf $2" - "}($1 ~ "Name"){print $2}' FS=":" "${M_JOB_CTRL}"
				else
					usage
				fi
				m_remove_run_files
				M_ACTION=NORUN
				return 0
				;;
			-lf)
				#======================================
				# Is this a list all or 'like' search?
				#======================================
				shift
				printf "%s\n" "${C_BREAK}"
				if [[ $# -gt 0 ]] 
				then
					[[ ${1} == "%"* ]] || m_fail 1 "Error:Pattern expected \"%pattern\"" 
					printf "%s\n" " Matching functions available in muse.funcs"
					printf "%s\n" "${C_BREAK}"
					sed -n '/## Name[\t]*:[\t]*m_/p' "${M_FUNCS}" | sed 's/\(## Name[\t]*:\)//g' | 
						sort | awk -v p=${1#\%*} '($1 ~ p){print}'
				else
					printf "%s\n" " Common functions available in muse.funcs"
					printf "%s\n" "${C_BREAK}"
					sed -n '/## Name[\t]*:[\t]*m_/p' "${M_FUNCS}" | sed 's/\(## Name[\t]*:\)//g' | sort
				fi
				m_remove_run_files
				printf "%s\n" "${C_BREAK}"
				M_ACTION=NORUN
				return 0
				;;
			-lp) ## List plugins
				#======================================
				# Is this a list all or 'like' search?
				#======================================
				shift
				printf "%s\n" "${C_BREAK}"
				if [[ $# -gt 0 ]] 
				then
					[[ ${1} == "%"* ]] || m_fail 1 "Error:Pattern expected \"%pattern\"" 
					printf "%s\n" " Matching plugins available"
					printf "%s\n" "${C_BREAK}"
					P="${1#\%*}"
					awk -v p=${P} ' (($NF !~ /.*[.]+.*/)&&($NF !~ /^ms_/)&&($NF ~ p)) \
						{printf "\t%s\n", $NF}' FS="/" ${M_MUSE_MAP} | sort
				else
					printf "%s\n" " Common plugins available"
					printf "%s\n" "${C_BREAK}"
					awk '(($NF ~ /^m_/)&&($NF !~ /.*[.]+.*/)&&($NF !~ /^ms_/)) \
						{printf "\t%s\n", $NF}' FS="/" ${M_MUSE_MAP} | sort
				fi
				m_remove_run_files
				printf "%s\n" "${C_BREAK}"
				M_ACTION=NORUN
				return 0
				;;
			-ls) ## List suite files
				#======================================
				# Is this a list all or 'like' search?
				#======================================
				shift
				printf "%s\n" "${C_BREAK}"
				if [[ $# -gt 0 ]] 
				then
					[[ ${1} == "%"* ]] || m_fail 1 "Error:Pattern expected \"%pattern\"" 
					printf "%s\n" " Matching suites available"
					printf "%s\n" "${C_BREAK}"
					awk -v p="${1#\%*}" '(($NF ~ p)&&($NF !~ /.*[.]+.*/)&&($NF !~ /m_/)) \
						{printf "\t%s\n", $NF;exit}' FS="/" ${M_MUSE_MAP} | sort
				else
					printf "%s\n" " Common suites available"
					printf "%s\n" "${C_BREAK}"
					awk '(($NF ~ /^ms_/)&&($NF !~ /.*[.]+.*/)&&($NF !~ /^m_/)) \
						{printf "\t%s\n", $NF}' FS="/" ${M_MUSE_MAP} | sort
				fi
				m_remove_run_files
				printf "%s\n" "${C_BREAK}"
				M_ACTION=NORUN
				return 0
				;;
			-s|--show) ## Show the current user tag
				m_remove_run_files
				printf "\n%s\n\n" "Current user tag - ${M_USR_TAG}"
				M_ACTION=NORUN
				return 0
				;;
			-t|--tag) ## Set a user tag
				shift
				mkdir "/tmp/${1}" 2>/dev/null || m_fail 1 "Error: Invalid tag name passed (${FUNCNAME})"  
				[[ -d "/tmp/${1}" ]] || m_fail 1 "Error: Failed to catch dir creation error (${FUNCNAME})" 
				rm -rf "/tmp/${1}" || m_fail 1 "Error: Failed to remove tmp dir (${FUNCNAME})" 
				echo "${1}" > "${M_USR_TAG_FILE}" || 
					m_fail 1 "Error: Failed to write tag to control file (${M_USR_TAG_FILE}) (${FUNCNAME})" 
				m_remove_run_files
				M_ACTION=NORUN
				return 0
				;;
			-vf) ## View a function in this file
				shift
				if [[ $# -eq 1 ]]
				then
					awk -v p=${1} -v f=1 '
						((f == 1) && (NR == FNR) && ($0 ~ /^## Name[\t]*:[\t ]*/) && ($NF == p)) {fs = (NR -2)}
						((f == 1) && (NR == FNR) && (fs != 0) && ($0 ~ /^}/)){fe = NR;f = 2}
						((NR != FNR) && (FNR >= fs) && (FNR <= fe)){print}
					' "${M_FUNCS}" "${M_FUNCS}"
				else
					usage
				fi
				m_remove_run_files
				M_ACTION=NORUN
				return 0
				;;
			-vif)
				shift
				if [[ $# -eq 1 ]]
				then
					if [[ -n $(egrep "^${1}" "${M_FUNCS}") ]]
					then
						vi +/^${1} "${M_FUNCS}"
					else
						echo "Function not found"
					fi
				else
					usage
				fi
				m_remove_run_files
				M_ACTION=NORUN
				return 0
				;;
			-vp|-vs) ## View a plugin
				shift
				[[ $# -eq 1 ]] || m_fail 1 "Error: Usage. Single arg expected (${FUNCNAME})" 
				FILE=$(awk -v p="${1}" '($NF == p){print;exit}' FS="/" ${M_MUSE_MAP})
				if [[ -r ${FILE} ]]
				then
					cat ${FILE}
				else
					echo "Plugin (${1}) not found"
				fi
				m_remove_run_files
				M_ACTION=NORUN
				return 0
				;;
			-vip|-vis) ## Edit a plugin
				shift
				[[ $# -eq 1 ]] || m_fail 1 "Error: Usage. Single arg expected (${FUNCNAME})" 
				FILE=$(awk -v p="${1}" '($NF == p){print;exit}' FS="/" ${M_MUSE_MAP})
				if [[ -r ${FILE} ]]
				then
					vi ${FILE}
				else
					echo "Plugin (${1}) not found"
				fi
				m_remove_run_files
				M_ACTION=NORUN
				return 0
				;;
			-x|--exec)echo "Option not coded yet";exit 1;;

			*)
				usage
				m_remove_run_files
				M_ACTION=NORUN
				return 0
				;;
		esac
	fi
	#======================================
	# No options passed so just straight job list
	#======================================
	m_process_cmd_list $@
	M_ACTION=RUN
}
##############################################################
##
## Name		:	m_process_cmd_list
## Author	:	Bradley Atkins
## Description	:	Called after options are parsed and 
##			we have a list of jobs to be run
## Date		:	28/08/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_process_cmd_list()
{
	[[ $# -eq 0 ]] && m_fail 1 "Error: Usage args (${FUNCNAME})"

	local LINE

	for ARG in $@
	do
		## Script name must begin m_ or ms_ and be followed by at least 1 alphanumeric char
		[[ "${ARG}" =~ ^(m_|ms_[[:alnum:]]+) ]] || m_fail 1 "Invalid script name (${ARG}) (${FUNCNAME})"

		## If its m_ just add it to the command line
		if [[ ${ARG} =~ ^(m_[[:alnum:]]+) ]]
		then
			# TODO CREATE A FUNCTION TO PROCESS bg AND wait ON CMD LINE

			m_add_job_to_control_file ${ARG} ${M_JOB_BG} ${M_JOB_WAIT} ${M_GRP_ID} 
		else
			m_expand_suite ${ARG}
		fi
	done

	#======================================
	# Create our run control files
	#======================================
	export M_RUN_LIST=$(m_split_file ${C_JOB_CTRL_FCNT} "${M_JOB_CTRL}" "${C_JOB_PREFIX}" "${M_CONTROL}")
	[[ -n "${M_RUN_LIST}" ]] || m_fail 1 "Error:Failed to set run list (${FUNCNAME})" 
	if [[ -s "${M_GRP_CTRL}" ]] 
	then
		export M_GRP_LIST=$(m_split_file ${C_GRP_CTRL_FCNT} "${M_GRP_CTRL}" "${C_GRP_PREFIX}" "${M_CONTROL}")
		[[ -n "${M_GRP_LIST}" ]] || m_fail 1 "Error:Failed to set group list (${FUNCNAME})" 
	fi
}
##############################################################
##
## Name		:	m_split_file
## Author	:	Bradley Atkins
## Description	:	More flexible version of bash split cmd
##			allows targeted output
## Date		:	29/08/2012
## Args		:	1 - Number of lines per op file
##			2 - ip file
##			3 - op file prefix
##			4 - op dir
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_split_file()
{
	## Usage
	[[ $# -eq 4 ]] || return 1
	## File Validation
	m_check_file -rs "${2}" s || return 2
	m_check_file -dw "${4}" s || return 3

	local LINE OPN=1 INDX=1 

	:>"${4}/${OPN}-${3}${M_RND}"

	while read LINE
	do
		printf "%s\n" "${LINE}" >> "${4}/${OPN}-${3}${M_RND}"
		INDX=$((INDX = $INDX + 1))
		if [[ ${INDX} -eq $(($1 + 1)) ]]
		then
			printf "%s " "${4}/${OPN}-${3}${M_RND}"
			OPN=$((OPN = $OPN + 1))
			INDX=1
		fi
	done < "${2}"
}
##############################################################
##
## Name		:	m_add_job_to_control_file
## Author	:	Bradley Atkins
## Description	:	Add a single job to the job control file.
##			As passed from command line parser.
## Date		:	28/08/2012
## Args		:	1 - A single muse job file
##			2 - BG flag Y/N
##			3 - WAIT flag Y/N
##			4 - GRP ID n/""
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_add_job_to_control_file()
{
	[[ $# -eq 3 || $# -eq 4 ]] || m_fail 1 "Error: Usage args (${FUNCNAME})"

	[[ -n "${M_JOB_CTRL}" ]] || m_fail 1 "Error: Pointer to run data file not set (${FUNCNAME})"  

	local FQFN RUN_HOST 

	#======================================
	# Does the job exist?
	#======================================
	
	FQFN=$(m_find_file "${1}") || m_fail 1 "Error:Failed to find file (${1}) (${FUNCNAME})" 
	m_check_file -rs "${FQFN}" s || m_fail 1 "Error Job file validation (${FQFN}) (${FUNCNAME})"

	m_parse_muse_script "${1}"

	{
		printf "%s\n" "ID:${M_JOB_ID}"
		printf "%s\n" "Name:${1}"
		printf "%s\n" "Location:${FQFN}"
		printf "%s\n" "RunHost:${M_JOB_HOST}"
		printf "%s\n" "Group:${4}"
		printf "%s\n" "Pid:"
		printf "%s\n" "Start:"
		printf "%s\n" "Finish:"
		printf "%s\n" "Wait:${3}"
		printf "%s\n" "Background:${2}"
	} >> "${M_JOB_CTRL}"

	M_JOB_ID=$((M_JOB_ID = M_JOB_ID + 1))

	M_JOB_COUNT=$(( $M_JOB_COUNT + 1 ))
}
##############################################################
##
## Name		:	m_expand_suite
## Author	:	Bradley Atkins
## Description	:	Expand a suite file into a list of m_ cmds
## Date		:	22/07/2012
## Args		:	1 - Name of suite file
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##
##############################################################
m_expand_suite()
{
	[[ $# -eq 1 || $# -eq 2 ]] || m_fail 1 "Error: Usage args (${FUNCNAME})"

	local JOB JOB_CNT=0 CMD_LIST FQSF SUITE="${1}" TMPFILE1="$(m_get_tmp_file ${FUNCNAME})" WAIT=N BG=N
	m_check_file -fwr "${TMPFILE1}" s || m_fail 1 "Error:Failed to create tmp file (${FUNCNAME})"

	FQSF=$(m_find_file "${SUITE}")
	m_check_file -rsf "${FQSF}" s || m_fail 1 "File (${SUITE}) failed permissions check -rsf (${FUNCNAME})"

	## Get rid of all the comments etc
	m_clean_file "${FQSF}" "${TMPFILE1}"
	[[ -n $(grep -w "${SUITE}" "${TMPFILE1}") ]] && m_fail 1 "Suite file (${FQSF}) is recursive (${FUNCNAME})"

	while read LINE
	do
		TMP=$(m_check_bg ${LINE}) 
		[[ $? -eq 0 ]] && BG=Y && LINE=${TMP}

		TMP=$(m_check_wait ${LINE}) 
		[[ $? -eq 0 ]] && WAIT=Y && LINE=${TMP}

		[[ "${LINE}" =~ ^(m_|ms_[[:alnum:]]+) ]] || m_fail 1 "Line begins with invalid script name (${LINE}) (${FUNCNAME})"
		
		#======================================
		# Single job on line or multiple?
		#======================================
		if [[ $(echo "${LINE}" | wc -w) -eq 1 ]]
		then
			#======================================
			# Single arg may be a job or suite
			#======================================
			if [[ ${LINE} =~ ^(ms_[[:alnum:]]+) ]]
			then
				m_expand_suite ${LINE}
			else
				m_add_job_to_control_file ${LINE} ${BG} ${WAIT} 
			fi
		else
			#======================================
			# Parallel jobs on one line 
			#======================================
			BG=Y
			for JOB in ${LINE}
			do
				[[ ${JOB} =~ ^(m_|ms_[[:alnum:]]+) ]] || 
					m_fail 1 "Error: Invalid Script Name (${JOB}) (${FUNCNAME})" 
				
				[[ ${JOB} =~ ^(ms_[[:alnum:]]+) ]] && 
					m_fail 1 "Error: Parallel suite files not supported (${FUNCNAME})" 
				
				m_add_job_to_control_file ${JOB} ${BG} ${WAIT} ${M_GRP_INDX}

				JOB_CNT=$((JOB_CNT = ${JOB_CNT} + 1))
			done

			m_add_grp_to_control_file ${WAIT} ${JOB_CNT}
			JOB_CNT=0
			M_GRP_INDX=$((M_GRP_INDX = $M_GRP_INDX + 1))
		fi
		WAIT=N
		BG=N
	done < "${TMPFILE1}"
}
##############################################################
##
## Name		:	m_add_grp_to_control_file
## Author	:	Bradley Atkins
## Description	:	Add a group to the master control file
##			background group jobs
## Date		:	30/08/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_add_grp_to_control_file()
{
	[[ $# -eq 2 ]] || m_fail 1 "Error: Usage args (${FUNCNAME})"

	[[ -n "${M_GRP_CTRL}" ]] || m_fail 1 "Error: Pointer to group data file not set (${FUNCNAME})"  

	{
		printf "%s\n" "ID:${M_GRP_INDX}"
		printf "%s\n" "Wait:${1}"
		printf "%s\n" "No of Jobs:${2}"
		printf "%s\n" "Pid List:"
	} >> "${M_GRP_CTRL}"
}
##############################################################
##
## Name		:	m_check_bg
## Author	:	Bradley Atkins
## Description	:	Helper script to check if a group or 
##			job runs in bg
## Date		:	30/08/2012
## Args		:	Line of jobs from suite file
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_check_bg()
{
	if [[ -n $(echo $@ | egrep '( BG|BG )') ]]	
	then
		echo "$@" | sed 's/BG//g;s/^[ \t]*//;s/[ \t]*$//'
		for s in ${PIPESTATUS[@]}; do [[ $s -eq 0 ]] || return 1; done
		return 0
	fi
	echo $@
	return 2
}
##############################################################
##
## Name		:	m_check_wait
## Author	:	Bradley Atkins
## Description	:	Helper script to check if a group or 
##			job requires a wait
## Date		:	30/08/2012
## Args		:	Line of jobs from suite file
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_check_wait()
{
	if [[ -n $(echo $@ | egrep '( WAIT|WAIT )') ]] 
	then
		echo "$@" | sed 's/WAIT//g;s/^[ \t]*//;s/[ \t]*$//'
		for s in ${PIPESTATUS[@]}; do [[ $s -eq 0 ]] || return 1; done
		return 0
	fi
	echo $@
	return 1
}
##############################################################
##
## Name		:	m_clean_file
## Author	:	Bradley Atkins
## Description	:	Strip out comments etc from a file
##			If source and dest are the same then overwrite
## Date		:	25/07/2012
## Args		:	1 - Fully qualified file name (source)
## 			2 - Fully qualified file name (output)
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_clean_file()
{
	[[ $# -eq 2 ]] || m_fail 1 "Error: Usage args (${FUNCNAME})"

	if [[ "${1}" == "${2}" ]] 
	then
		local FILE="$(m_get_tmp_file ${FUNCNAME})" 
		m_check_file -frw "${FILE}" s || m_fail 1 "Error:Failed to create tmp file (${FUNCNAME})"
		m_check_file -frw "${1}" s || m_fail 1 "Error: File validation (${1}) 1 (${FUNCNAME})" 
		sed 's/[#].*$//g' "${1}" | awk NF > "${FILE}"
		for s in ${PIPESTATUS[@]}; do [[ $s -eq 0 ]] || m_fail 1 "Error:Pipe failed 1 (${FUNCNAME})" ; done 
		cat "${FILE}" > "${1}"
	else
		m_check_file -fr "${1}" s || m_fail 1 "Error: File validation (${1}) 2 (${FUNCNAME})" 
		m_check_file -fw "${2}" s || m_fail 1 "Error: File validation (${2}) 3 (${FUNCNAME})" 
		sed 's/[#].*$//g' "${1}" | awk NF > "${2}"
		for s in ${PIPESTATUS[@]}; do [[ $s -eq 0 ]] || m_fail 1 "Error:Pipe failed 2 (${FUNCNAME})" ; done 
	fi
}
##############################################################
##
## Name		:	m_validate_options
## Author	:	Bradley Atkins
## Description	:	
## Date		:	21/07/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##
##############################################################
m_validate_options()
{

	[[ $# -eq 0 ]] && return 1
	local OPT TEST=pass OPTIONS=$(echo "${1}" | tr -d "-" | sed 's/[.]*/ /g')
	[[ -z "${OPTIONS}" ]] && return 2

	for OPT in ${OPTIONS}
	do
		[[ "${OPT}" =~ ([${C_VALID_OPTIONS}]+) ]] || TEST=fail
		if [[ ${TEST} == fail ]] 
		then
			echo ${OPT}
			return 3
		fi
	done

	return 0
}
##############################################################
##
## Name		:	m_find_file
## Author	:	Bradley Atkins
## Description	:	Search down from muse root for a named
##			flat file. 
## Date		:	20/07/2012
## Args		:	1 - file name
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##
##############################################################
m_find_file()
{
	[[ -n "${1}" ]] || return 1

	local FILE="$(awk -v f=${1} '($NF == f){print $0;exit}' FS="/" ${M_MUSE_MAP})"

	[[ -z "${FILE}" ]] && return 2

	echo "${FILE}"

	return 0
}
##############################################################
##
## Name		:	m_map_muse
## Author	:	Bradley Atkins
## Description	:	Map all muse files to our map file
## Date		:	18/08/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_map_muse()
{
	[[ -n "${M_DEV_ROOT}" && -d "${M_DEV_ROOT}" ]] || m_fail 1 "Dev root not found (${FUNCNAME})"
	[[ -n "${M_ROOT}" ]] || m_fail 1 "M_ROOT not set: (${FUNCNAME})"

	# TODO ADD CHECK FOR MUSE_DEV GROUP WHEN GROUPS SET UP
	find "${M_DEV_ROOT}" -type f > "${M_MUSE_MAP}" || m_fail 1 "Error: Write to muse map failed 1 (${FUNCNAME})" 
	find "${M_ROOT}" -type f >> "${M_MUSE_MAP}" || m_fail 1 "Error: Write to muse map failed 2 (${FUNCNAME})"
}
##############################################################
##
## Name		:	m_check_file
## Author	:	Bradley Atkins
## Description	:	Check a file's properties
## Date		:	20/07/2012
## Args		:	1 - Properties eg -frwx
##			2 - fully qualified name
##			3 - [v] verbose [s] silent
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##
##############################################################
m_check_file()
{
	[[ $# -ne 3 ]] && return 1
	[[ -n ${2} ]] || return 2
	[[ "${1}" =~ ^(-[frxwdabcgGhkLNOpsStu]+$) ]] || return 3
	[[ ${3} =~ ^([vs])+$ ]] || return 4

	local MODE=${3}

	for OPTION in $(echo "${1}" | tr -d "-" | sed 's/[.]*/ /g')
	do
		if ! eval [[ "-${OPTION}" "${2}" ]]
		then
			[[ ${3} == v ]] &&  echo "-${OPTION}"
			return 5
		fi
	done
}
##############################################################
##
## Name		:	m_print_marker
## Author	:	Bradley Atkins
## Description	:	Print a standard marker to stdout. This is 
##			used to deliniate sections of output for 
##			post processing.
## Date		:	19/08/2012
## Args		:	1 - A single recognised valid marker char
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_print_marker()
{
	[[ $# -eq 1 ]] || m_fail 1 "Error: Usage args (${FUNCNAME})"

	case ${1} in
		"+") printf "%s\n" "${C_PLUS}";;
		"-") printf "%s\n" "${C_MINUS}";;
		"_") printf "%s\n" "${C_USCORE}";;
		"=") printf "%s\n" "${C_EQUALS}";;		## TODO EXCLUSION MARKER
		".") printf "%s\n" "${C_EVIDENCE}";;		## Evidence Marker
		"@") printf "%s\n" "${C_ERROR}";;		## Error Marker

		*) m_fail 1 "Unexpected char passed (${1}) (${FUNCNAME})";;
	esac
}
##############################################################
##
## Name		:	m_print_master_header
## Author	:	Bradley Atkins
## Description	:	Print a header for the master stdout
## Date		:	29/09/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_print_master_header()
{
	printf "%s\n" "${C_BREAK}"
	printf "%s\n" " Status:	Commencing MUSE run"
	printf "%s\n" " Start:         $(date)"
	printf "%s\n" " User Tag:      ${M_USR_TAG}"
	printf "%s\n" " Sys  Tag:      $(echo ${M_SYS_TAG} | sed 's/^[.]//g')"
	printf "%s\n" "${C_BREAK}"
}
##############################################################
##
## Name		:	m_print_master_footer
## Author	:	Bradley Atkins
## Description	:	Print a footer for the master stdout
## Date		:	29/09/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_print_master_footer()
{
	printf "%s\n" "${C_BREAK}"
	printf "%s\n" " Status:	Completed"
	printf "%s\n" " Start:         $(date)"
	printf "%s\n" "${C_BREAK}"
}
##############################################################
##
## Name		:	m_print_results_header
## Author	:	Bradley Atkins
## Description	:	Print a header for a results file
## Date		:	25/08/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_print_results_header()
{
	printf "%s\n" "${C_BREAK}"
	printf "%s\n" " Start:          ${M_JOB_START}"
	printf "%s\n" " User:           ${USER}"
	printf "%s\n" " Job:            ${M_JOB_NAME}"
	printf "%s\n" " Master:         ${M_MASTER_HOST}"
	printf "%s\n" " Slave:          ${M_JOB_HOST}"
	printf "%s\n" " User Tag:       ${M_USR_TAG}"
	printf "%s\n" " Sys  Tag:       $(echo ${M_SYS_TAG} | sed 's/^[.]//g')"
	printf "%s\n" "${C_BREAK}"
}
##############################################################
##
## Name		:	m_print_results_trailer
## Author	:	Bradley Atkins
## Description	:	Print a trailer for a results file
## Date		:	25/08/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_print_results_trailer()
{
	printf "%s\n" "${C_BREAK}"
	printf "%s\n" " Finish:	${M_JOB_FINISH}"
	printf "%s\n" " Duration:	${M_JOB_DURATION}"
	printf "%s\n" "${C_BREAK}"
}
##############################################################
##
## Name		:	m_remove_run_files
## Author	:	Bradley Atkins
## Description	:	After a non executing command we don't
##			want to keep our runtime files and dirs
## Date		:	27/08/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_remove_run_files()
{
	[[ -d "${M_RUN_DIR}" ]] && rm -rf "${M_RUN_DIR}/"
}
##############################################################
##
## Name		:	m_get_exp_file
## Author	:	Bradley Atkins
## Description	:	Create an expected results file with a unique name
## Date		:	28/08/2012
## Args		:	1 - Name of function or script requiring file
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_get_exp_file()
{
	[[ $# -eq 1 ]] || m_fail 1 "Error: Usage args (${1}) calling (${FUNCNAME})"

	local FILE="${M_NEWEXP}/${1}.exp"

	:>"${FILE}" || m_fail 1 "Error: Couldn't write to exp file(${FILE}) (${FUNCNAME})" 

	echo "${FILE}"
}
##############################################################
##
## Name		:	m_get_tmp_file
## Author	:	Bradley Atkins
## Description	:	Create a tmp file with a unique name
## Date		:	28/08/2012
## Args		:	Name of function or script requiring file
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_get_tmp_file()
{
	## USage
	[[ $# -eq 1 ]] || return 1

	local FILE="${M_TMP}/${M_FILE_INDX}_${1}_${M_RND}.tmp"

	## Instantiate it with a null write
	:>"${FILE}" || return 2

	echo "${FILE}"

	M_FILE_INDX=$((M_FILE_INDX = $M_FILE_INDX + 1))
}
##############################################################
##
## Name		:	m_check_state
## Author	:	Bradley Atkins
## Description	:	Check our msg files for the given state
##			Return name/s of msg files with that state
## Date		:	16/09/2012
## Args		:	1 - STATE
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_check_state()
{
	# TODO This assumes that the listener will update the state
	# this needs to change 
	[[ $# -eq 1 ]] || m_fail 1 "Error: Usage args (${FUNCNAME})"

	local MSG_FILE MSG_LIST LIST STATE

	MSG_LIST=$(ls ${M_CONTROL}/*-m_msg_data_* 2>/dev/null | tr "\n" " ") 

	for MSG_FILE in ${MSG_LIST}
	do
		STATE=$(cat ${MSG_FILE} | sed -n '/Status:/p' | cut -d":" -f2)
		[[ ${STATE} == ${1} ]] && LIST="${LIST} ${MSG_FILE}"
	done

	echo "${LIST}"
}

##############################################################
##
## Name		:	m_get_msgs
## Author	:	Bradley Atkins
## Description	:	Check the msg files for new msgs
## Date		:	16/09/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_get_msgs()
{
	[[ -n ${M_MSG_QUEUE} ]] || m_fail 1 "Error: Message Queue file not defined (${FUNCNAME})" 
	:>"${M_MSG_QUEUE}"

	local MSG_FILE_LIST MSG_FILE ACTION RV=1

	MSG_FILE_LIST=$(ls "${M_CONTROL}"/[[:digit:]]*-m_msg_data_* 2>/dev/null | tr "\n" " ") 
	[[ -n "${MSG_FILE_LIST}" ]] || m_fail 1 "Error: No msg files found (${FUNCNAME})" 

	for MSG_FILE in ${MSG_FILE_LIST}
	do
		ACTION=$(cat ${MSG_FILE} | sed -n '/Action:/p' | cut -d":" -f2)

		case ${ACTION} in
			Read)
				echo "${MSG_FILE}|$(tail -1 ${MSG_FILE})" >> "${M_MSG_QUEUE}" || 
					m_fail 1 "Error: Failed to write msg to msg queue file (${M_MSG_QUEUE}) (${FUNCNAME})" 
				m_write_msg_field ${C_MSG_ACTION} "Skip"
				RV=0
				;;
			Skip) 
				continue
				;;
			*) 
				m_fail 1 "Error: Unrecognised action (${CMD}) in msg file (${MSG_FILE}) (${FUNCNAME})"
				;; 
		esac
	done

	return ${RV}
}
##############################################################
##
## Name		:	m_prioritise_msgs
## Author	:	Bradley Atkins
## Description	:	Process the list of msgs returned by m_get_msgs
##			and convert status to a list of prioritised actions for
##			the pseudo state machine
##			Priorities -
##			1 - COMPLETED
##			2 - FAILED
##			3 - FATAL
##			4 - MANUAL
## Date		:	21/10/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_prioritise_msgs()
{
	local TMPFILE1="$(m_get_tmp_file ${FUNCNAME})" LINE P=0
	[[ -f ${TMPFILE1} ]] || m_fail 1 "Error: Failed to create tmp file (${FUNCNAME})" 

	m_check_file -rw "${M_MSG_QUEUE}" s || m_fail 1 "Error: Permissions (${M_MSG_QUEUE}) (${FUNCNAME})" 

	{
		while read LINE 
		do
			P=0

			STATUS=$(echo ${LINE} | awk '{print $2}' FS="Status=")
			[[ -n "${STATUS}" ]] || m_fail 1 "Error: Status not found (${FUNCNAME})" 
		
			case ${STATUS} in
				COMPLETED) P=1;;
				FAILED) P=2;;
				FATAL) P=3;;
				MANUAL) P=4;;
				*) m_fail 1 "Error: Unrecognised Status in line (${LINE}) (${FUNCNAME})";; 
			esac

			echo "${P}|${LINE}"
		done < "${M_MSG_QUEUE}"
	
	} | sort -t "|" -k 1 > "${TMPFILE1}"

	cp "${TMPFILE1}" "${M_MSG_QUEUE}"
}
##############################################################
##
## Name		:	m_process_msg_queue
## Author	:	Bradley Atkins
## Description	:	Process the message queue and take action
## Date		:	22/10/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_process_msg_queue()
{
	local TMPFILE1="$(m_get_tmp_file ${FUNCNAME})" LINE CTL_FILE STATUS PIPE PID JOB_FILE TO=${C_MSG_PIPE_TO}
	[[ -f ${TMPFILE1} ]] || m_fail 1 "Error: Failed to create tmp file (${FUNCNAME})" 

	m_check_file -frw "${M_MSG_QUEUE}" s || m_fail 1 "Error: Msg queue not found (${FUNCNAME})" 

	while read LINE
	do
		#======================================
		# Split the line and get the control file
		# 	and the status field
		#======================================
		CTL_FILE="$(echo ${LINE} | cut -d"|" -f2)"
		STATUS="$(echo ${LINE} | cut -d"|" -f3 | awk '{print $NF}' FS="Status=" )"
		[[ (-n ${STATUS}) && (-n ${CTL_FILE}) ]] || 
			m_fail 1 "Error: Failed to parse msg ctl file (${FUNCNAME})" 
		PIPE=${CTL_FILE##*/}

		case ${STATUS} in
			COMPLETED)
				#======================================
				# Completed job so update its control file
				# with the finish time
				# tell the listener to close
				# then delete the pipe
				#======================================
				JOB_FILE="$(sed -n '/^JobFile:/p' "${CTL_FILE}" | cut -d":" -f 2)"
				[[ -n ${JOB_FILE} ]] || m_fail 1 "Error: Failed to retrieve job ctl file (${FUNCNAME})" 
				m_check_file -frw "${JOB_FILE}" s || m_fail 1 "Error: job ctl validation failure (${FUNCNAME})" 
				PID="$(sed -n '/^Pid:/p' "${CTL_FILE}" | cut -d":" -f 2)"
				[[ ${PID} =~ ^[[:digit:]]+$ ]] || m_fail 1 "Error: PID validation (${FUNCNAME})" 

				m_write_job_field ${C_JOB_FINISH} "$(date)" "${JOB_FILE}"

				m_send_msg_to_pipe "FINISH" ${FUNCNAME} "${PIPE}"
	
				while [[ $(( TO = TO - 1 )) -ge 0 ]]
				do
					ps -o pid= -o comm= -p ${PID} | grep u_listener > "${TMPFILE1}"
					[[ -s "${TMPFILE1}" ]] || break	
					sleep 1
				done
				[[ ${TO} -ge 0 ]] || m_fail 1 "Error: Time out waiting for listener to exit (${FUNCNAME})" 

				[[ -p ${PIPE} ]] && rm -f ${PIPE} 2>/dev/null
				;;
			FAILED)
				#======================================
				# Log the finish time in the ctl file
				# Log the error in the run log
				# Close the pipe
				#======================================

				;;
				
			FATAL)
				#======================================
			        # Log the finish time in the ctl file
			        # Log the error in the run log
			        # Close the pipe
				# Flag the fatal error so we can close up 
				# on exiting this function after processing all msgs
			        #======================================
				;;
			MANUAL)
				#======================================
				# Manual intervention
				# Echo the manual request to stdout
				# Flag request so we can wait for user input
				# after processing all msgs
				#======================================

				;;
			*);;
		esac
	done < "${M_MSG_QUEUE}"

}
##############################################################
##
## Name		:	m_get_file_data
## Author	:	Bradley Atkins
## Description	:	Clean the output of the find | ls command
##			Cut out the time stamps etc just leaving
##			perms owner grp size and path. First two params
##			are optional. Complexity arises from requirement to 
##			handle files and dirs with escape sequences and spaces
##			in the names.
## Date		:	19/09/2012
## Args		:	1 - -s Optional Sort by file or directory name
##			2 - -d Optional find of directories (Defaults to files)
##			3 - -min Optional min depth for find
##			4 - -max Optional max depth for find
##			5 - -f Option field list. Defaults to "1 3 4 5 9"
##			6 - Top level directory for find
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_get_file_data()
{
	[[ $# > 7 ]] && m_fail 1 "Error: Usage args (${FUNCNAME})"  

	local SORT="cat" OPT="-l" TYPE="f" MIN_D= MAX_D= FL="1 3 4 5 9"

	while [[ $# > 1 ]]
	do
		case ${1} in
			-s) SORT="sort -k5";;
			-f) TYPE="f";;
			-d) TYPE="d";OPT="-ld";;
			-min)
				shift 
				[[ ${1} =~ ^[[:digit:]]+$ ]] || 
					m_fail 1 "Error: Min depth param (${FUNCNAME})" 
				MIN_D="-mindepth ${1}"
				;;
			-max)
				shift 
				[[ ${1} =~ ^[[:digit:]]+$ ]] || 
					m_fail 1 "Error: Max depth param (${FUNCNAME})"
				MAX_D="-maxdepth ${1}"
				;;
			*) m_fail 1 "Error: Unrecognised option (${1}) (${FUNCNAME})";;
		esac
		shift
	done

	[[ -d "${1}" ]] || m_fail 1 "Error: Not a directory (${1}) (${FUNCNAME})" 

	awk '{
		lo=""
		for (i = 1; i <= NF; i++){
			if (i <= 5)
				if (i != 2)
					lo = lo" "$i

			if ($i ~ /\//){
				sub(/ */, " ", $0)			
				printf lo" "$0"\n"
				next
			}
			$i = ""
		}
	}' <(find "${1}" ${MIN_D} ${MAX_D} -type ${TYPE} | sed 's#\\#\\\\#g' | xargs -i ls ${OPT} "{}") | ${SORT}
}
##############################################################
##
## Name		:	m_get_epoch_seconds
## Author	:	Bradley Atkins
## Description	:	Return a date time as epoch seconds
## Date		:	24/09/2012
## Args		:	1- Optional time string, e.g. -
##			"Jan 1, 1980 00:00:01"
##			"January 1, 1980 23:59:59"
##			"January 1 1980 23:59:59"
##			"1980/1/1 00:00:01"
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_get_epoch_seconds()
{
	[[ $# -gt 1 ]] && return 1

	local TIME="$(date +%s)"

	[[ $# -eq 1 ]] && TIME=$(date +%s -d "${1}" 2>/dev/null)   
	[[ $? -eq 0 ]] || return 2

	echo "${TIME}"
}
##############################################################
##
## Name		:	m_get_duration
## Author	:	Bradley Atkins
## Description	:	Return the duration (diff) between a start
##			and an end time.
## Date		:	26/09/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_get_duration()
{
	[[ $# -eq 2 ]] || m_fail 1 "Error: Usage args (${FUNCNAME})"
	
	local TM1="${1}" TM2="${2}" SECS1 SECS2 SECS

	SECS1=$(m_get_epoch_seconds "${TM1}") || m_fail 1 "Error: Incorrect time format for conversion (${TM1}) (${FUNCNAME})"
	SECS2=$(m_get_epoch_seconds "${TM2}") || m_fail 1 "Error: Incorrect time format for conversion (${TM2}) (${FUNCNAME})"

	[[ ${SECS2} -ge ${SECS1} ]] || m_fail 1 "Error: Negative time (${FUNCNAME})" 

	SECS=$(( SECS2 - SECS1 ))

	M_JOB_DURATION=$(printf ""%dh:%dm:%ds"\n" $((${SECS}/3600)) $((${SECS}%3600/60)) $((${SECS}%60)))
}
##############################################################
##
## Name		:	m_get_date_from_epoch
## Author	:	Bradley Atkins
## Description	:	Convert an epoch time string (secs) to
##			a human readable date format.
## Date		:	24/09/2012
## Args		:	1 - Valid date format string
##			2 - Time (Epoch Seconds) '@nnnnnnnn'
##			e.g. -
##			"%d/%m/%Y" "@2147483647"
##			"%d/%m/%Y %H:%M:%S" "@2147483647"
##			"%d/%m/%y" "@2147483647"
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_get_date_from_epoch()
{
	## Usage
	[[ $# -eq 2 ]] || return 1
	
	local FORMAT= EPOCH=

	[[ $# -eq 2 ]] && FORMAT="+\"${1}\""

	[[ ${2} =~ ^[@]+[[:digit:]]+$ ]] || return 2

	eval date --date="${2}" "${FORMAT}" || return 3
}
##############################################################
##
## Name		:	m_get_hms
## Author	:	Bradley Atkins
## Description	:	Convert epoch seconds to HMS
## Date		:	26/09/2012
## Args		:	1 - Epoch secs
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_get_hms()
{
	[[ $# -eq 1 ]] || m_fail 1 "Error: Usage args (${FUNCNAME})"
	[[ ${1} =~ ^[[:digit:]]+$ ]] || m_fail 1 "Error: Usage format (${FUNCNAME})" 

	printf ""%dh:%dm:%ds"\n" $(($SECS/3600)) $(($SECS%3600/60)) $(($SECS%60))
}
##############################################################
##
## Name		:	m_set_runtime_env
## Author	:	Bradley Atkins
## Description	:	Create a file containing our environment
##			variables so they can be sourced
## Date		:	25/08/2012
## Args		:	
## Status	:	Reviewed 	[n]
##			Tested 		[n]
##			Released 	[n]
##############################################################
m_set_runtime_env()
{
	set -a 

	#======================================
	# We will be creating our env file under
	# the run root so we need to establish
	# that first of all.
	#======================================
	M_RUN_ROOT="${M_RUN_ROOT:-/tmp/muse}"				## Top level run dir
	M_RUN="${M_RUN_ROOT}/run"
	M_CONTROL_FILES="${M_RUN_ROOT}/m_system"			## Control files for the current run

	M_USR_TAG_FILE="${M_CONTROL_FILES}/m_usr_tag_file"		## The tag file
	[[ -z "${M_USR_TAG}" ]] && m_get_usr_tag			## Might already be set by ssh call
	M_USR_TAG="${M_USR_TAG:-default}"				## The user tag
	M_SYS_TAG="${M_SYS_TAG:-$(date +"%y%m%d%H%M%S")}"		## The system tag

	M_RND=${M_RND:-$RANDOM}						## Random number for tmp file names etc
	M_RUN_DIR="${M_RUN}/${M_USR_TAG}/${M_SYS_TAG}"			## Tagged run dir
	M_TMP="${M_RUN_DIR}/m_tmp"					## Tmp files created during run
	M_NEWEXP="${M_RUN_DIR}/m_newexp"				## New expected files created during run
	M_RESULTS="${M_RUN_DIR}/m_results"				## Results dir
	M_CONTROL="${M_RUN_DIR}/m_control"				## The control files for run
	M_PID_FILE="${M_CONTROL}/m_pid_file"				## Record the PID
	M_PID=$$							## The PID muse is running under
	M_GRP_CTRL="${M_GRP_DATA:-$M_CONTROL/m_grp_ctrl_$M_RND}"	## Main control file for the scripts to be run
	M_JOB_CTRL="${M_JOB_CTRL:-$M_CONTROL/m_job_ctrl_$M_RND}"	## Main control file for the scripts to be run
	M_RUN_LOG="${M_CONTROL}/m_run_log"				## Log file for run
	M_JOB_FILE=							## Point to current job control file
	M_JOB_ID=${M_JOB_ID:-1}						## Job control fields
	M_JOB_NAME=
	M_JOB_PATH=
	M_JOB_HOST=$(hostname)
	M_JOB_GRP=
	M_JOB_PID=
	M_JOB_FINISH=
	M_JOB_WAIT=${M_JOB_WAIT:-N}
	M_JOB_BG=${M_JOB_BG:-N}
	M_JOB_FINISH=
	M_JOB_START=
	M_JOB_DURATION=

	M_ERRORS="${M_RESULTS}/errors.log"				## Summary of failures and warnings
	M_EVIDENCE="${M_RESULTS}/evidence.log"				## Summary of marked evidence

	M_LAST_FAIL_MSG=						## Set to the last failure
	M_MUSE_MAP="${M_CONTROL}/m_muse_map"				## Map of all muse files. Saves repeated finds

	M_DEV_ROOT="${HOME}/.muse_root"					## Dev directory for muse_dev group

	M_CLUSTER=home							## Cluster Identifier TODO NEEDS A FUNCTION
	M_MASTER_HOST=${M_MASTER_HOST:-$M_JOB_HOST}			## The Master server

	M_FILE_INDX=1							## Unique index for tmp files
	M_GRP_INDX=1							## Unique index for parallel job groups
	M_GRP_ID=							## Index of current active group
	M_BG=N								## Flag if wait after parallel jobs initiated
	M_GRP_WAIT=N							## Flag if we need to wait for a bg job or group
	M_PID_LIST=							## List of BG pids for wait function
	M_GRP_PID_LIST=							## List of group BG pids for wait function
	M_JOB_COUNT=0							## The number of jobs being run
	M_GRP_COUNT=0							## The number of jobs as read from the grp file
	M_OUT=								## The out file for stdout of current job
	M_MUSE_STATUS=${M_MUSE_STATUS}					## Master or slave. Set by muse

	M_MSG_CTL=							## Current msg control file
	M_MSG_NAME=							## Msg control file fields
	M_MSG_PID=							## PID if job is background
	M_MSG_JOB_CTRL_FILE=
	M_MSG_JOB_HOST=							## Name of the slave
	M_MSG_START=
	M_MSG_FINISH=
	M_MSG_STATE=							## Like COMPLETED FAILED etc.
	M_MSG_PIPE=${M_MSG_PIPE}					## Name of named pipe for job
	M_MSG_STATUS=							## Slave status
	M_MSG_WRITE_CNT=1						## Write counter for msg ctl file
	M_MSG_ACTION="Skip"						## Action flag for state machine. Skip or Read msg
	M_MSG_QUEUE=							## File holding the message queue

	M_ACTION=							## Action for state machine

	TS=								## Job ctrl timestamp

	GE=								## Global Error MSG for debug

	set +a
}
